# -*- coding: utf-8 -*-
import scrapy
from scrapy.loader import ItemLoader
from ScrapyArticle.items import JobBoleArticleItem
from ScrapyArticle.items import ArticleItemLoader

class BoleSpider(scrapy.Spider):
    name = 'bole'
    allowed_domains = ['python.jobbole.com']
    start_urls = ['http://python.jobbole.com/all-posts/']

    def parse(self, response):
        """
                1. 获取文章列表页中的文章url并交给scrapy下载后并进行解析
                2. 获取下一页的url并交给scrapy进行下载， 下载完成后交给parse
        """
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")

           # yield scrapy.Request(url=post_url, meta={"front_image_url":image_url}, callback=self.parse_detail_item_loder)
            yield scrapy.Request(url=post_url, meta={"front_image_url": image_url},callback=self.parse_detail_image)
        # 提取下一页并交给scrapy进行下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield scrapy.Request(url=next_url, callback=self.parse)
    def parse_detail(self, response):
        article_item = JobBoleArticleItem()
        article_item["front_image_url"] = response.meta.get("front_image_url", "")  # 文章封面图

        article_item['name'] = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")
        article_item['create_date'] = response.xpath("//p[@class='entry-meta-hide-on-mobile']/text()").extract()[0].strip().encode("utf-8").replace("·",
                                                                                                                    "").strip()
        article_item['praise_nums'] = response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract()[0]
        article_item['fav_nums'] = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract()[0]
        article_item['comment_nums'] = response.xpath("//a[@href='#article-comment']/span/text()").extract()[0]
        #article_item['content'] = response.xpath("//div[@class='entry']").extract()[0]
        tag_list = response.xpath("//p[@class='entry-meta-hide-on-mobile']/a/text()").extract()
        tag_list = [element for element in tag_list if not element.strip().encode("utf-8").endswith("评论")]
        article_item['tags'] = ",".join(tag_list)
        article_item['url']=response.url
        yield article_item

    def parse_detail_item_loder(self,response):
        # 通过item loader加载item
        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        item_loader = ArticleItemLoader(item=JobBoleArticleItem(), response=response)
        item_loader.add_css("name", ".entry-header h1::text")
        item_loader.add_value("url", response.url)

        item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_value("front_image_url", [front_image_url])

        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", ".bookmark-btn::text")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")


        article_item = item_loader.load_item()
        yield article_item

    # 自定义图片自动下载方法
    def parse_detail_image(self,response):
        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        article_item = JobBoleArticleItem()
        article_item["front_image_url"]=[front_image_url]  #用scrapy自带的图片保存方法，这个地方必须是list集合
        yield article_item


